#include "../include/PageLib.hpp"
#include "../include/tinyxml2.h"
#include "../include/CppJieba.hpp"
#include "../include/DirScanner.hpp"
#include <iostream>
#include <string>
#include <regex>
#include <vector>
#include <fstream>
#include <map>
#include <memory>

using std::cout;
using std::endl;
using std::string;
using std::vector;
using std::ofstream;
using std::map;
using std::shared_ptr;

using namespace tinyxml2;

//现在Conf的指针和Dirs的对象需要外部提供****
PageLib::PageLib(Configuration* conf)/* FileProcessor& fileProcessor) */
: _conf(conf)
{
    _conf->getInstance();
    map<string, string> mp = _conf->getConfigMap();
    string xmlFilesPath = mp.find("xml")->second;

    shared_ptr<DirScanner> dirs(new DirScanner());
    dirs->operator()(xmlFilesPath);
    _files = dirs->files();
}



void PageLib::create(){
    int docid = 1;

    CppJieba jieba;

    for(auto & file: _files){
        XMLDocument doc;
        doc.LoadFile(file.c_str());
        if(doc.ErrorID())
        {
            perror("PageLib:: create doc.LoadFile ");
            return;
        }


        XMLElement *itemNode = 
            doc.FirstChildElement("rss")->FirstChildElement("channel")->FirstChildElement("item");
        while(itemNode)
        {
            string title = itemNode->FirstChildElement("title")->GetText();
            /* cout << "title: " << title << endl; */
            string url=  itemNode->FirstChildElement("link")->GetText();
            /* cout << "url: " << url << endl; */
            // 如果没有content就用description代替
            string content;
            if(itemNode->FirstChildElement("content:encoded")){
                content = itemNode->FirstChildElement("content:encoded")->GetText();
            }
            else if(itemNode->FirstChildElement("content")){
                content = itemNode->FirstChildElement("content")->GetText();
            }
            else if(itemNode->FirstChildElement("description")){
                content = itemNode->FirstChildElement("description")->GetText();
            }
            else{// 没有内容，直接跳过这篇文章
                itemNode = itemNode->NextSiblingElement("item");
                continue;
            }


            std::regex reg("<[^>]+>");//通用正则表达式过滤html
            content = regex_replace(content, reg, "");
            /* cout << "content: " << content << endl; */
        
            WebPage wp(_conf, jieba, docid, title, url, content);
            //构造webpage时需要先用jieba分词，再用configuration里面的stopwords成员，去除停用词
            //最后的结果，分成员存入webpage

            _webPages.push_back(std::move(wp));

            ++docid;
            itemNode = itemNode->NextSiblingElement("item");
        }

    }
}

vector<WebPage> & PageLib::getWebPages(){
    return _webPages;
}

